{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PyTorch：控制流和参数共享"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "作为动态图和权重共享的一个例子，我们实现了一个非常奇怪的模型：一个全连接的ReLU网络，在每一次前向传播时，它的隐藏层的层数为随机1到4之间的数，这样可以多次重用相同的权重来计算。\n",
    "\n",
    "因为这个模型可以使用普通的Python流控制来实现循环，并且我们可以通过在定义转发时多次重用同一个模块来实现最内层之间的权重共享。\n",
    "\n",
    "我们利用Mudule的子类很容易实现这个模型："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 649.90625\n",
      "1 649.3474731445312\n",
      "2 645.8781127929688\n",
      "3 665.173583984375\n",
      "4 640.8623046875\n",
      "5 614.624267578125\n",
      "6 641.2725219726562\n",
      "7 631.5670776367188\n",
      "8 637.7459716796875\n",
      "9 625.2185668945312\n",
      "10 470.83551025390625\n",
      "11 435.3675231933594\n",
      "12 632.5811767578125\n",
      "13 345.1820068359375\n",
      "14 611.3906860351562\n",
      "15 563.5171508789062\n",
      "16 627.3644409179688\n",
      "17 625.0584716796875\n",
      "18 621.886962890625\n",
      "19 587.0455322265625\n",
      "20 612.4263305664062\n",
      "21 142.03684997558594\n",
      "22 598.9332275390625\n",
      "23 111.41944122314453\n",
      "24 94.00847625732422\n",
      "25 530.2828369140625\n",
      "26 564.1966552734375\n",
      "27 500.838134765625\n",
      "28 56.085899353027344\n",
      "29 462.2825622558594\n",
      "30 439.5290222167969\n",
      "31 385.6937561035156\n",
      "32 359.0207214355469\n",
      "33 363.9302978515625\n",
      "34 424.4493103027344\n",
      "35 391.6949157714844\n",
      "36 358.6614685058594\n",
      "37 320.9956359863281\n",
      "38 221.74868774414062\n",
      "39 205.73878479003906\n",
      "40 236.0008544921875\n",
      "41 381.0203857421875\n",
      "42 287.8484802246094\n",
      "43 139.3165740966797\n",
      "44 159.48472595214844\n",
      "45 248.602294921875\n",
      "46 287.087646484375\n",
      "47 238.6629638671875\n",
      "48 295.69122314453125\n",
      "49 252.0790557861328\n",
      "50 242.408935546875\n",
      "51 177.77325439453125\n",
      "52 108.4017333984375\n",
      "53 168.93350219726562\n",
      "54 94.99005889892578\n",
      "55 75.32698059082031\n",
      "56 274.39697265625\n",
      "57 152.6976776123047\n",
      "58 89.20470428466797\n",
      "59 42.248905181884766\n",
      "60 52.67405319213867\n",
      "61 76.1695327758789\n",
      "62 158.10206604003906\n",
      "63 122.26139068603516\n",
      "64 98.58473205566406\n",
      "65 159.34042358398438\n",
      "66 98.90213012695312\n",
      "67 71.44273376464844\n",
      "68 72.49298095703125\n",
      "69 41.42448043823242\n",
      "70 34.17814254760742\n",
      "71 78.87855529785156\n",
      "72 55.21364212036133\n",
      "73 41.58122634887695\n",
      "74 102.9864501953125\n",
      "75 40.64149475097656\n",
      "76 40.934425354003906\n",
      "77 35.26224899291992\n",
      "78 31.285680770874023\n",
      "79 21.238710403442383\n",
      "80 19.18100929260254\n",
      "81 53.7769889831543\n",
      "82 25.61323356628418\n",
      "83 37.243507385253906\n",
      "84 25.85808563232422\n",
      "85 79.48888397216797\n",
      "86 34.117366790771484\n",
      "87 41.64287567138672\n",
      "88 21.483169555664062\n",
      "89 19.96319007873535\n",
      "90 12.65923023223877\n",
      "91 20.127750396728516\n",
      "92 11.75792121887207\n",
      "93 47.72924041748047\n",
      "94 8.95595932006836\n",
      "95 6.40992546081543\n",
      "96 28.215970993041992\n",
      "97 26.285930633544922\n",
      "98 25.037487030029297\n",
      "99 10.359728813171387\n",
      "100 14.697885513305664\n",
      "101 21.69222068786621\n",
      "102 20.50885009765625\n",
      "103 8.85484504699707\n",
      "104 6.457906723022461\n",
      "105 10.724051475524902\n",
      "106 7.964795112609863\n",
      "107 14.662099838256836\n",
      "108 12.277120590209961\n",
      "109 8.473353385925293\n",
      "110 5.86277437210083\n",
      "111 4.270740509033203\n",
      "112 4.854345321655273\n",
      "113 9.213086128234863\n",
      "114 8.4703950881958\n",
      "115 45.34480667114258\n",
      "116 4.933383941650391\n",
      "117 23.948394775390625\n",
      "118 11.79050350189209\n",
      "119 19.32332992553711\n",
      "120 8.267915725708008\n",
      "121 18.042495727539062\n",
      "122 3.6004364490509033\n",
      "123 4.625056743621826\n",
      "124 10.145763397216797\n",
      "125 27.155025482177734\n",
      "126 13.962530136108398\n",
      "127 10.88161563873291\n",
      "128 3.9266014099121094\n",
      "129 5.087352752685547\n",
      "130 4.946312427520752\n",
      "131 53.5407829284668\n",
      "132 1.7588692903518677\n",
      "133 8.060940742492676\n",
      "134 15.258874893188477\n",
      "135 20.368417739868164\n",
      "136 22.749622344970703\n",
      "137 5.636255741119385\n",
      "138 6.03618860244751\n",
      "139 9.295034408569336\n",
      "140 2.611478805541992\n",
      "141 21.7631893157959\n",
      "142 19.991914749145508\n",
      "143 7.710391998291016\n",
      "144 5.3223395347595215\n",
      "145 10.8408784866333\n",
      "146 10.279752731323242\n",
      "147 19.00205421447754\n",
      "148 23.98591423034668\n",
      "149 3.8788182735443115\n",
      "150 12.624588012695312\n",
      "151 10.515420913696289\n",
      "152 23.420185089111328\n",
      "153 6.857257843017578\n",
      "154 6.93535852432251\n",
      "155 9.844923973083496\n",
      "156 5.3539204597473145\n",
      "157 7.847427845001221\n",
      "158 7.260693073272705\n",
      "159 9.215002059936523\n",
      "160 3.2031781673431396\n",
      "161 7.949552536010742\n",
      "162 6.199731349945068\n",
      "163 12.309928894042969\n",
      "164 5.850223541259766\n",
      "165 3.9574220180511475\n",
      "166 12.0177583694458\n",
      "167 4.221400737762451\n",
      "168 4.744832515716553\n",
      "169 9.23845386505127\n",
      "170 12.06603717803955\n",
      "171 8.128302574157715\n",
      "172 5.9497294425964355\n",
      "173 10.188653945922852\n",
      "174 9.788980484008789\n",
      "175 6.970864295959473\n",
      "176 2.825300455093384\n",
      "177 2.15470027923584\n",
      "178 26.835355758666992\n",
      "179 2.137932062149048\n",
      "180 3.276779890060425\n",
      "181 2.1044960021972656\n",
      "182 2.036688804626465\n",
      "183 48.60462951660156\n",
      "184 5.671703338623047\n",
      "185 12.845856666564941\n",
      "186 24.100217819213867\n",
      "187 13.610061645507812\n",
      "188 6.109210968017578\n",
      "189 18.53804588317871\n",
      "190 9.154383659362793\n",
      "191 6.597208499908447\n",
      "192 3.627408981323242\n",
      "193 10.33941650390625\n",
      "194 2.2116825580596924\n",
      "195 5.042715549468994\n",
      "196 1.4549392461776733\n",
      "197 30.961841583251953\n",
      "198 4.96574068069458\n",
      "199 3.1533620357513428\n",
      "200 5.18477725982666\n",
      "201 5.538798809051514\n",
      "202 138.5970001220703\n",
      "203 7.53177547454834\n",
      "204 15.403565406799316\n",
      "205 44.91699981689453\n",
      "206 35.465049743652344\n",
      "207 28.529964447021484\n",
      "208 17.795278549194336\n",
      "209 25.532976150512695\n",
      "210 2.873838186264038\n",
      "211 16.18170928955078\n",
      "212 38.8431510925293\n",
      "213 19.58062744140625\n",
      "214 13.159402847290039\n",
      "215 4.692615509033203\n",
      "216 1.5346240997314453\n",
      "217 7.405038833618164\n",
      "218 51.28666687011719\n",
      "219 13.856213569641113\n",
      "220 11.423957824707031\n",
      "221 2.4745872020721436\n",
      "222 37.250614166259766\n",
      "223 2.056067705154419\n",
      "224 1.553322434425354\n",
      "225 31.805994033813477\n",
      "226 3.9840261936187744\n",
      "227 6.65031623840332\n",
      "228 7.128425121307373\n",
      "229 7.4032673835754395\n",
      "230 4.933427333831787\n",
      "231 3.3975932598114014\n",
      "232 4.582798480987549\n",
      "233 4.429608345031738\n",
      "234 4.782041072845459\n",
      "235 2.0663318634033203\n",
      "236 1.9860395193099976\n",
      "237 1.8377052545547485\n",
      "238 6.898558616638184\n",
      "239 1.7485443353652954\n",
      "240 5.952418327331543\n",
      "241 4.6291279792785645\n",
      "242 2.9404242038726807\n",
      "243 3.4712066650390625\n",
      "244 3.5378494262695312\n",
      "245 13.432295799255371\n",
      "246 4.827559471130371\n",
      "247 4.075112342834473\n",
      "248 13.052265167236328\n",
      "249 5.499269962310791\n",
      "250 4.0897369384765625\n",
      "251 9.547341346740723\n",
      "252 2.388657331466675\n",
      "253 7.4311699867248535\n",
      "254 4.298134803771973\n",
      "255 6.485882759094238\n",
      "256 2.5496060848236084\n",
      "257 2.049370527267456\n",
      "258 9.744486808776855\n",
      "259 1.4428943395614624\n",
      "260 1.132689356803894\n",
      "261 2.239455223083496\n",
      "262 2.0907580852508545\n",
      "263 6.747678279876709\n",
      "264 1.81243097782135\n",
      "265 4.040063858032227\n",
      "266 3.1702730655670166\n",
      "267 2.4151854515075684\n",
      "268 2.570096969604492\n",
      "269 2.451326370239258\n",
      "270 2.6437578201293945\n",
      "271 1.2390302419662476\n",
      "272 2.127716302871704\n",
      "273 1.8945773839950562\n",
      "274 1.906104564666748\n",
      "275 1.3636077642440796\n",
      "276 1.762778639793396\n",
      "277 1.7875449657440186\n",
      "278 1.3717775344848633\n",
      "279 0.9009202718734741\n",
      "280 2.0402822494506836\n",
      "281 0.6947506666183472\n",
      "282 1.386694073677063\n",
      "283 0.6883621215820312\n",
      "284 1.5611732006072998\n",
      "285 2.235590934753418\n",
      "286 1.9316482543945312\n",
      "287 0.9180230498313904\n",
      "288 1.6861330270767212\n",
      "289 0.6836841106414795\n",
      "290 0.9312564730644226\n",
      "291 1.0811858177185059\n",
      "292 0.9257139563560486\n",
      "293 1.697833776473999\n",
      "294 1.5615261793136597\n",
      "295 0.7501528263092041\n",
      "296 0.8612582087516785\n",
      "297 2.479653835296631\n",
      "298 0.7534535527229309\n",
      "299 1.0887576341629028\n",
      "300 0.8893488645553589\n",
      "301 1.2226345539093018\n",
      "302 0.8907167911529541\n",
      "303 0.6699960231781006\n",
      "304 0.52253657579422\n",
      "305 0.548001766204834\n",
      "306 2.615400552749634\n",
      "307 0.683580219745636\n",
      "308 1.8504219055175781\n",
      "309 1.92081618309021\n",
      "310 0.8692836761474609\n",
      "311 0.8028703927993774\n",
      "312 0.3033757209777832\n",
      "313 1.2408654689788818\n",
      "314 1.5181127786636353\n",
      "315 0.8011281490325928\n",
      "316 1.036556363105774\n",
      "317 2.8236732482910156\n",
      "318 0.571832537651062\n",
      "319 1.2624156475067139\n",
      "320 2.8791496753692627\n",
      "321 0.8841482996940613\n",
      "322 0.5844407081604004\n",
      "323 0.7054100632667542\n",
      "324 0.6677689552307129\n",
      "325 1.2639350891113281\n",
      "326 8.210532188415527\n",
      "327 0.9301782250404358\n",
      "328 4.32470703125\n",
      "329 7.837436676025391\n",
      "330 1.1307899951934814\n",
      "331 6.026207447052002\n",
      "332 0.431029349565506\n",
      "333 0.3128054440021515\n",
      "334 0.20965014398097992\n",
      "335 0.17973396182060242\n",
      "336 0.20642368495464325\n",
      "337 4.518723964691162\n",
      "338 0.09610443562269211\n",
      "339 0.1490473747253418\n",
      "340 0.9067457914352417\n",
      "341 2.417705774307251\n",
      "342 0.4502405524253845\n",
      "343 1.2925455570220947\n",
      "344 0.31089136004447937\n",
      "345 0.96807861328125\n",
      "346 2.709595203399658\n",
      "347 0.6634021401405334\n",
      "348 1.7690403461456299\n",
      "349 0.8778661489486694\n",
      "350 0.3714772164821625\n",
      "351 1.1572186946868896\n",
      "352 0.3056812882423401\n",
      "353 1.8493183851242065\n",
      "354 0.385202020406723\n",
      "355 0.6225449442863464\n",
      "356 0.7323828935623169\n",
      "357 2.910461187362671\n",
      "358 0.7926974892616272\n",
      "359 1.8166849613189697\n",
      "360 3.2478978633880615\n",
      "361 0.9482606053352356\n",
      "362 0.5038476586341858\n",
      "363 0.7932608127593994\n",
      "364 1.2517428398132324\n",
      "365 14.852334976196289\n",
      "366 1.0575411319732666\n",
      "367 2.6036980152130127\n",
      "368 3.5891456604003906\n",
      "369 12.215752601623535\n",
      "370 41.82831954956055\n",
      "371 2.2563953399658203\n",
      "372 11.504471778869629\n",
      "373 25.934513092041016\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "374 98.9085922241211\n",
      "375 11.944602966308594\n",
      "376 1.074958324432373\n",
      "377 131.6365966796875\n",
      "378 101.91561889648438\n",
      "379 42.543540954589844\n",
      "380 7.2745256423950195\n",
      "381 39.885032653808594\n",
      "382 18.664888381958008\n",
      "383 78.31068420410156\n",
      "384 112.29149627685547\n",
      "385 11.854430198669434\n",
      "386 20.250028610229492\n",
      "387 22.358028411865234\n",
      "388 33.73491287231445\n",
      "389 40.6268424987793\n",
      "390 43.36037063598633\n",
      "391 37.51011657714844\n",
      "392 14.644497871398926\n",
      "393 20.882680892944336\n",
      "394 24.085248947143555\n",
      "395 13.153386116027832\n",
      "396 24.084169387817383\n",
      "397 11.854269027709961\n",
      "398 6.942853927612305\n",
      "399 8.112987518310547\n",
      "400 11.231542587280273\n",
      "401 9.785791397094727\n",
      "402 15.568102836608887\n",
      "403 5.834254264831543\n",
      "404 4.510124206542969\n",
      "405 4.452259540557861\n",
      "406 12.16606616973877\n",
      "407 11.24567985534668\n",
      "408 3.9041590690612793\n",
      "409 9.848668098449707\n",
      "410 4.092050552368164\n",
      "411 2.86269211769104\n",
      "412 3.7501535415649414\n",
      "413 6.764093399047852\n",
      "414 4.039853572845459\n",
      "415 4.459043979644775\n",
      "416 2.7198143005371094\n",
      "417 1.673218011856079\n",
      "418 7.936428546905518\n",
      "419 5.9317803382873535\n",
      "420 1.6249287128448486\n",
      "421 4.568819999694824\n",
      "422 3.0434658527374268\n",
      "423 2.6088833808898926\n",
      "424 3.6907284259796143\n",
      "425 4.341337203979492\n",
      "426 3.0388286113739014\n",
      "427 2.022979497909546\n",
      "428 2.1867799758911133\n",
      "429 2.025862455368042\n",
      "430 1.7667962312698364\n",
      "431 5.221782207489014\n",
      "432 1.4519035816192627\n",
      "433 2.067483901977539\n",
      "434 1.4759918451309204\n",
      "435 1.3351768255233765\n",
      "436 2.794762134552002\n",
      "437 4.134113311767578\n",
      "438 1.987613320350647\n",
      "439 1.4986746311187744\n",
      "440 2.5960023403167725\n",
      "441 2.118567705154419\n",
      "442 2.0146663188934326\n",
      "443 2.791696310043335\n",
      "444 0.8737941384315491\n",
      "445 1.1011663675308228\n",
      "446 3.3540689945220947\n",
      "447 1.6380727291107178\n",
      "448 0.9606466293334961\n",
      "449 0.5747268199920654\n",
      "450 0.774868905544281\n",
      "451 5.120718479156494\n",
      "452 0.3833392262458801\n",
      "453 1.132341742515564\n",
      "454 0.46730223298072815\n",
      "455 0.5068734288215637\n",
      "456 2.475135326385498\n",
      "457 3.3571012020111084\n",
      "458 0.7236261963844299\n",
      "459 0.6042624115943909\n",
      "460 1.8193122148513794\n",
      "461 4.315296173095703\n",
      "462 1.167815089225769\n",
      "463 2.2287285327911377\n",
      "464 0.41986531019210815\n",
      "465 2.512904644012451\n",
      "466 4.255651950836182\n",
      "467 0.6602839827537537\n",
      "468 1.2193858623504639\n",
      "469 0.2669932544231415\n",
      "470 2.1419100761413574\n",
      "471 1.8807512521743774\n",
      "472 1.5272468328475952\n",
      "473 1.315522313117981\n",
      "474 1.249306082725525\n",
      "475 0.8069095015525818\n",
      "476 2.760483741760254\n",
      "477 0.9791107177734375\n",
      "478 0.25420594215393066\n",
      "479 0.8756033778190613\n",
      "480 1.4460184574127197\n",
      "481 0.7776396870613098\n",
      "482 0.5898275375366211\n",
      "483 0.7964105606079102\n",
      "484 1.4469711780548096\n",
      "485 0.5500622987747192\n",
      "486 2.379448890686035\n",
      "487 0.2653568387031555\n",
      "488 0.318904846906662\n",
      "489 0.4955824017524719\n",
      "490 0.5617710947990417\n",
      "491 0.5568835735321045\n",
      "492 0.6085115671157837\n",
      "493 0.49452152848243713\n",
      "494 0.32701602578163147\n",
      "495 0.17530936002731323\n",
      "496 0.9090036749839783\n",
      "497 0.7687430381774902\n",
      "498 0.5676519274711609\n",
      "499 0.11646056920289993\n"
     ]
    }
   ],
   "source": [
    "# 可运行代码见本文件夹中的 dynamic_net.py\n",
    "import random\n",
    "import torch\n",
    "\n",
    "class DynamicNet(torch.nn.Module):\n",
    "    def __init__(self, D_in, H, D_out):\n",
    "        \"\"\"\n",
    "        在构造函数中，我们构造了三个nn.Linear实例，它们将在前向传播时被使用。\n",
    "        \"\"\"\n",
    "        super(DynamicNet, self).__init__()\n",
    "        self.input_linear = torch.nn.Linear(D_in, H)\n",
    "        self.middle_linear = torch.nn.Linear(H, H)\n",
    "        self.output_linear = torch.nn.Linear(H, D_out)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        对于模型的前向传播，我们随机选择0、1、2、3，\n",
    "        并重用了多次计算隐藏层的middle_linear模块。\n",
    "        由于每个前向传播构建一个动态计算图，\n",
    "        我们可以在定义模型的前向传播时使用常规Python控制流运算符，如循环或条件语句。\n",
    "        在这里，我们还看到，在定义计算图形时多次重用同一个模块是完全安全的。\n",
    "        这是Lua Torch的一大改进，因为Lua Torch中每个模块只能使用一次。\n",
    "        \"\"\"\n",
    "        h_relu = self.input_linear(x).clamp(min=0)\n",
    "        for _ in range(random.randint(0, 3)):\n",
    "            h_relu = self.middle_linear(h_relu).clamp(min=0)\n",
    "        y_pred = self.output_linear(h_relu)\n",
    "        return y_pred\n",
    "\n",
    "\n",
    "# N是批大小；D是输入维度\n",
    "# H是隐藏层维度；D_out是输出维度\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# 产生输入和输出随机张量\n",
    "x = torch.randn(N, D_in)\n",
    "y = torch.randn(N, D_out)\n",
    "\n",
    "# 实例化上面定义的类来构造我们的模型\n",
    "model = DynamicNet(D_in, H, D_out)\n",
    "\n",
    "# 构造我们的损失函数（loss function）和优化器（Optimizer）。\n",
    "# 用平凡的随机梯度下降训练这个奇怪的模型是困难的，所以我们使用了momentum方法。\n",
    "criterion = torch.nn.MSELoss(reduction='sum')\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=1e-4, momentum=0.9)\n",
    "for t in range(500):\n",
    "    \n",
    "    # 前向传播：通过向模型传入x计算预测的y。\n",
    "    y_pred = model(x)\n",
    "\n",
    "    # 计算并打印损失\n",
    "    loss = criterion(y_pred, y)\n",
    "    print(t, loss.item())\n",
    "\n",
    "    # 清零梯度，反向传播，更新权重 \n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (Spyder)",
   "language": "python3",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "227.797px"
   },
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
